{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Install  the  packages\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "UGOkPYguQ2z-"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eRjCooE9IBAX",
        "outputId": "1fae5f9b-a24f-459d-ac36-f8eb391838bb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m27.4/27.4 MB\u001b[0m \u001b[31m13.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m30.5/30.5 MB\u001b[0m \u001b[31m12.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pypdf\n",
            "  Downloading pypdf-5.1.0-py3-none-any.whl.metadata (7.2 kB)\n",
            "Requirement already satisfied: typing_extensions>=4.0 in /usr/local/lib/python3.10/dist-packages (from pypdf) (4.12.2)\n",
            "Downloading pypdf-5.1.0-py3-none-any.whl (297 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m298.0/298.0 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pypdf\n",
            "Successfully installed pypdf-5.1.0\n",
            "Requirement already satisfied: sentence-transformers in /usr/local/lib/python3.10/dist-packages (3.2.1)\n",
            "Requirement already satisfied: transformers<5.0.0,>=4.41.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (4.46.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (4.66.6)\n",
            "Requirement already satisfied: torch>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (2.5.1+cu121)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.5.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.13.1)\n",
            "Requirement already satisfied: huggingface-hub>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (0.26.2)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (11.0.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers) (3.16.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers) (2024.10.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers) (24.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers) (6.0.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers) (2.32.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers) (4.12.2)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers) (3.4.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers) (3.1.4)\n",
            "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers) (1.13.1)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy==1.13.1->torch>=1.11.0->sentence-transformers) (1.3.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.41.0->sentence-transformers) (1.26.4)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.41.0->sentence-transformers) (2024.9.11)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.41.0->sentence-transformers) (0.4.5)\n",
            "Requirement already satisfied: tokenizers<0.21,>=0.20 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.41.0->sentence-transformers) (0.20.3)\n",
            "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence-transformers) (1.4.2)\n",
            "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence-transformers) (3.5.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.11.0->sentence-transformers) (3.0.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.20.0->sentence-transformers) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.20.0->sentence-transformers) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.20.0->sentence-transformers) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.20.0->sentence-transformers) (2024.8.30)\n",
            "Collecting unstructured\n",
            "  Downloading unstructured-0.16.8-py3-none-any.whl.metadata (24 kB)\n",
            "Requirement already satisfied: chardet in /usr/local/lib/python3.10/dist-packages (from unstructured) (5.2.0)\n",
            "Collecting filetype (from unstructured)\n",
            "  Downloading filetype-1.2.0-py2.py3-none-any.whl.metadata (6.5 kB)\n",
            "Collecting python-magic (from unstructured)\n",
            "  Downloading python_magic-0.4.27-py2.py3-none-any.whl.metadata (5.8 kB)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from unstructured) (5.3.0)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from unstructured) (3.9.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from unstructured) (2.32.3)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.12.3)\n",
            "Collecting emoji (from unstructured)\n",
            "  Downloading emoji-2.14.0-py3-none-any.whl.metadata (5.7 kB)\n",
            "Collecting dataclasses-json (from unstructured)\n",
            "  Downloading dataclasses_json-0.6.7-py3-none-any.whl.metadata (25 kB)\n",
            "Collecting python-iso639 (from unstructured)\n",
            "  Downloading python_iso639-2024.10.22-py3-none-any.whl.metadata (13 kB)\n",
            "Collecting langdetect (from unstructured)\n",
            "  Downloading langdetect-1.0.9.tar.gz (981 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m981.5/981.5 kB\u001b[0m \u001b[31m17.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: numpy<2 in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.26.4)\n",
            "Collecting rapidfuzz (from unstructured)\n",
            "  Downloading rapidfuzz-3.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (11 kB)\n",
            "Collecting backoff (from unstructured)\n",
            "  Downloading backoff-2.2.1-py3-none-any.whl.metadata (14 kB)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.12.2)\n",
            "Collecting unstructured-client (from unstructured)\n",
            "  Downloading unstructured_client-0.28.1-py3-none-any.whl.metadata (20 kB)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.16.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.66.6)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from unstructured) (5.9.5)\n",
            "Collecting python-oxmsg (from unstructured)\n",
            "  Downloading python_oxmsg-0.0.1-py3-none-any.whl.metadata (5.0 kB)\n",
            "Requirement already satisfied: html5lib in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.1)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->unstructured) (2.6)\n",
            "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json->unstructured)\n",
            "  Downloading marshmallow-3.23.1-py3-none-any.whl.metadata (7.5 kB)\n",
            "Collecting typing-inspect<1,>=0.4.0 (from dataclasses-json->unstructured)\n",
            "  Downloading typing_inspect-0.9.0-py3-none-any.whl.metadata (1.5 kB)\n",
            "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.10/dist-packages (from html5lib->unstructured) (1.16.0)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.10/dist-packages (from html5lib->unstructured) (0.5.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (2024.9.11)\n",
            "Collecting olefile (from python-oxmsg->unstructured)\n",
            "  Downloading olefile-0.47-py2.py3-none-any.whl.metadata (9.7 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (2024.8.30)\n",
            "Collecting aiofiles>=24.1.0 (from unstructured-client->unstructured)\n",
            "  Downloading aiofiles-24.1.0-py3-none-any.whl.metadata (10 kB)\n",
            "Requirement already satisfied: cryptography>=3.1 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured) (43.0.3)\n",
            "Requirement already satisfied: eval-type-backport<0.3.0,>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured) (0.2.0)\n",
            "Requirement already satisfied: httpx>=0.27.0 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured) (0.27.2)\n",
            "Collecting jsonpath-python<2.0.0,>=1.0.6 (from unstructured-client->unstructured)\n",
            "  Downloading jsonpath_python-1.0.6-py3-none-any.whl.metadata (12 kB)\n",
            "Requirement already satisfied: nest-asyncio>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured) (1.6.0)\n",
            "Requirement already satisfied: pydantic<2.10.0,>=2.9.2 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured) (2.9.2)\n",
            "Requirement already satisfied: pypdf>=4.0 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured) (5.1.0)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured) (2.8.2)\n",
            "Requirement already satisfied: requests-toolbelt>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from unstructured-client->unstructured) (1.0.0)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/dist-packages (from cryptography>=3.1->unstructured-client->unstructured) (1.17.1)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.27.0->unstructured-client->unstructured) (3.7.1)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx>=0.27.0->unstructured-client->unstructured) (1.0.7)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.27.0->unstructured-client->unstructured) (1.3.1)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx>=0.27.0->unstructured-client->unstructured) (0.14.0)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->unstructured) (24.2)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<2.10.0,>=2.9.2->unstructured-client->unstructured) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.23.4 in /usr/local/lib/python3.10/dist-packages (from pydantic<2.10.0,>=2.9.2->unstructured-client->unstructured) (2.23.4)\n",
            "Collecting mypy-extensions>=0.3.0 (from typing-inspect<1,>=0.4.0->dataclasses-json->unstructured)\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl.metadata (1.1 kB)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.12->cryptography>=3.1->unstructured-client->unstructured) (2.22)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx>=0.27.0->unstructured-client->unstructured) (1.2.2)\n",
            "Downloading unstructured-0.16.8-py3-none-any.whl (1.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m63.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
            "Downloading dataclasses_json-0.6.7-py3-none-any.whl (28 kB)\n",
            "Downloading emoji-2.14.0-py3-none-any.whl (586 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m586.9/586.9 kB\u001b[0m \u001b[31m33.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading filetype-1.2.0-py2.py3-none-any.whl (19 kB)\n",
            "Downloading python_iso639-2024.10.22-py3-none-any.whl (274 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m274.9/274.9 kB\u001b[0m \u001b[31m20.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading python_magic-0.4.27-py2.py3-none-any.whl (13 kB)\n",
            "Downloading python_oxmsg-0.0.1-py3-none-any.whl (31 kB)\n",
            "Downloading rapidfuzz-3.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m58.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading unstructured_client-0.28.1-py3-none-any.whl (62 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.9/62.9 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading aiofiles-24.1.0-py3-none-any.whl (15 kB)\n",
            "Downloading jsonpath_python-1.0.6-py3-none-any.whl (7.6 kB)\n",
            "Downloading marshmallow-3.23.1-py3-none-any.whl (49 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.5/49.5 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
            "Downloading olefile-0.47-py2.py3-none-any.whl (114 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.6/114.6 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Building wheels for collected packages: langdetect\n",
            "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for langdetect: filename=langdetect-1.0.9-py3-none-any.whl size=993222 sha256=0de5007c21affc6fe5be588a8f8884cba29a9828c1e3065b5b73c8b2d1c3dd6a\n",
            "  Stored in directory: /root/.cache/pip/wheels/95/03/7d/59ea870c70ce4e5a370638b5462a7711ab78fba2f655d05106\n",
            "Successfully built langdetect\n",
            "Installing collected packages: filetype, rapidfuzz, python-magic, python-iso639, olefile, mypy-extensions, marshmallow, langdetect, jsonpath-python, emoji, backoff, aiofiles, typing-inspect, python-oxmsg, unstructured-client, dataclasses-json, unstructured\n",
            "Successfully installed aiofiles-24.1.0 backoff-2.2.1 dataclasses-json-0.6.7 emoji-2.14.0 filetype-1.2.0 jsonpath-python-1.0.6 langdetect-1.0.9 marshmallow-3.23.1 mypy-extensions-1.0.0 olefile-0.47 python-iso639-2024.10.22 python-magic-0.4.27 python-oxmsg-0.0.1 rapidfuzz-3.10.1 typing-inspect-0.9.0 unstructured-0.16.8 unstructured-client-0.28.1\n",
            "Collecting ctransformers\n",
            "  Downloading ctransformers-0.2.27-py3-none-any.whl.metadata (17 kB)\n",
            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from ctransformers) (0.26.2)\n",
            "Requirement already satisfied: py-cpuinfo<10.0.0,>=9.0.0 in /usr/local/lib/python3.10/dist-packages (from ctransformers) (9.0.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->ctransformers) (3.16.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->ctransformers) (2024.10.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->ctransformers) (24.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->ctransformers) (6.0.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->ctransformers) (2.32.3)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->ctransformers) (4.66.6)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->ctransformers) (4.12.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->ctransformers) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->ctransformers) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->ctransformers) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->ctransformers) (2024.8.30)\n",
            "Downloading ctransformers-0.2.27-py3-none-any.whl (9.9 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.9/9.9 MB\u001b[0m \u001b[31m56.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: ctransformers\n",
            "Successfully installed ctransformers-0.2.27\n"
          ]
        }
      ],
      "source": [
        "!pip install --quiet openai langchain\n",
        "!pip install --quiet -U lancedb\n",
        "!pip install pypdf\n",
        "!pip install sentence-transformers\n",
        "!pip install unstructured\n",
        "!pip install ctransformers"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Data Loading"
      ],
      "metadata": {
        "id": "yHyzaL_qbu4w"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Download [data](https://github.com/akashAD98/llam2_with_lancedb_/tree/main/multiple_pdf) and upload it in your google drive to run this example.\n",
        "While this step is in process,\n",
        "Take a break ☕\n",
        "\n",
        "Once this step is done, Now you are ready to move with next steps"
      ],
      "metadata": {
        "id": "SerUFcr9arNg"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Mount your drive for uploading data & model"
      ],
      "metadata": {
        "id": "Tbcx0cXSQ8PU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount(\"/content/drive\")"
      ],
      "metadata": {
        "id": "e-Usavvchqtm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "create folder inside drive & navigate to it\n",
        "while creating this example folder with name `llm` inside google drive is used.\n",
        "\n"
      ],
      "metadata": {
        "id": "TYXKyGqvdStz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/drive/llm"
      ],
      "metadata": {
        "id": "6Vbev0MgdPkr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Import Packages"
      ],
      "metadata": {
        "id": "8p8ggl_XZ56g"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import lancedb\n",
        "import re\n",
        "import pickle\n",
        "import requests\n",
        "import zipfile\n",
        "from pathlib import Path\n",
        "from langchain.document_loaders import UnstructuredHTMLLoader\n",
        "from langchain.embeddings import OpenAIEmbeddings\n",
        "from langchain.embeddings import HuggingFaceEmbeddings\n",
        "from langchain.document_loaders import PyPDFLoader, DirectoryLoader\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain.llms import CTransformers\n",
        "from langchain.chains import RetrievalQA\n",
        "from langchain.vectorstores import LanceDB\n",
        "from langchain.document_loaders import TextLoader\n",
        "from langchain.text_splitter import CharacterTextSplitter"
      ],
      "metadata": {
        "id": "CEOZFyNEinYw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's load the embedding model\n",
        "The all-mpnet-base-v2 model provides the best quality, while all-MiniLM-L6-v2 is 5 times faster and still offers good quality. you can try different models\n",
        "we are using all -MiniLM-L6-V2 models for this demo, & we are using cpu for inference"
      ],
      "metadata": {
        "id": "BJO6pcBXRFRK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# load embedding model\n",
        "\n",
        "embeddings_mini = HuggingFaceEmbeddings(\n",
        "    model_name=\"sentence-transformers/all-MiniLM-L6-v2\", model_kwargs={\"device\": \"cpu\"}\n",
        ")"
      ],
      "metadata": {
        "id": "IhUkKPE3jc71"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "select anyone below method"
      ],
      "metadata": {
        "id": "IB4nKLLCSscE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the  multiple pdfs\n",
        "pdf_folder_path = \"/content/drive/llm/pdf_paths/\"\n",
        "\n",
        "from langchain.document_loaders import PyPDFDirectoryLoader\n",
        "\n",
        "loader = PyPDFDirectoryLoader(pdf_folder_path)\n",
        "docs = loader.load()"
      ],
      "metadata": {
        "id": "PzQqTDX3RD-Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# # load single pdf\n",
        "# from langchain.document_loaders import PyPDFLoader\n",
        "# loader = PyPDFLoader(\"/content/pdfs/layoutparser.pdf\")\n",
        "# pages = loader.load_and_split()"
      ],
      "metadata": {
        "id": "D_gdAL1Wjf4c"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        " Now that we have our raw documents loaded, we need to pre-process them to generate embeddings:\n",
        "\n",
        "\n",
        "\n",
        " Initialize a RecursiveCharacterTextSplitter to split text documents into smaller chunks.\n",
        " This helps in processing large text data efficiently."
      ],
      "metadata": {
        "id": "YT2fJIyBSzSU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "text_splitter = RecursiveCharacterTextSplitter(\n",
        "    chunk_size=200,\n",
        "    chunk_overlap=50,\n",
        ")\n",
        "documents = text_splitter.split_documents(docs)\n",
        "embeddings = embeddings_mini"
      ],
      "metadata": {
        "id": "hYwyV99Jji_8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Lets use lanceDB as vectore store database & crete temp table"
      ],
      "metadata": {
        "id": "xbGpYTajS5Cf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "db = lancedb.connect(\"/tmp/lancedb\")\n",
        "table = db.create_table(\n",
        "    \"pandas_docs\",\n",
        "    data=[\n",
        "        {\n",
        "            \"vector\": embeddings.embed_query(\"Hello World\"),\n",
        "            \"text\": \"Hello World\",\n",
        "            \"id\": \"1\",\n",
        "        }\n",
        "    ],\n",
        "    mode=\"overwrite\",\n",
        ")\n",
        "docsearch = LanceDB.from_documents(documents, embeddings, connection=table)"
      ],
      "metadata": {
        "id": "-5hPRjS9jlUS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now, as we are using the Llama 2 quantized model, you need to download and upload it to your Google Drive. You can choose different versions based on your system specifications. For demonstration purposes, we are using a small model, but feel free to explore other options\n",
        "download the models from Thebloke repo.\n",
        "\n",
        "\n",
        "https://huggingface.co/TheBloke/Llama-2-7B-GGML/tree/main\n",
        "\n",
        "download the .bin file & kept in your current directory"
      ],
      "metadata": {
        "id": "XKwken-bb8nB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Loading the model\n",
        "def load_llm():\n",
        "    # Load the locally downloaded model here\n",
        "    llm = CTransformers(\n",
        "        model=\"llama-2-7b-chat.ggmlv3.q2_K.bin\",\n",
        "        model_type=\"llama\",\n",
        "        max_new_tokens=512,\n",
        "        temperature=0.6,\n",
        "    )\n",
        "    return llm\n",
        "\n",
        "\n",
        "qa = RetrievalQA.from_chain_type(\n",
        "    llm=load_llm(), chain_type=\"stuff\", retriever=docsearch.as_retriever()\n",
        ")"
      ],
      "metadata": {
        "id": "3A8Ct9QqjoIq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "You can ask the quetions to your data & i will give you response"
      ],
      "metadata": {
        "id": "H6zvmlYlTFIe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "query = \"What is DIA?\"\n",
        "qa.run(query)"
      ],
      "metadata": {
        "id": "WCRPud93jqz3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "query = \"What is LayoutParser ?\"\n",
        "qa.run(query)"
      ],
      "metadata": {
        "id": "y4RY2KAqjs3N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "I8-2QzE2TENM"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}